package com.timeriver.data_preprocess

import org.apache.spark.sql.{DataFrame, SparkSession}

/**
  * describe()/summary() 函数计算数据基本特征
  *   count：可以查看缺失值
  *   mean：平均值
  *   stddev：标准偏差
  *   min：最小值
  *   max：最大值
  */
object DescriptionDemo {
  def main(args: Array[String]): Unit = {
    val session: SparkSession = SparkSession.builder()
      .appName("基本数据统计")
      .master("local[6]")
      .getOrCreate()

    val df: DataFrame = session.read
      .format("jdbc")
      .option("url", "jdbc:mysql://localhost:3306/ml_datasets")
      .option("driver", "com.mysql.jdbc.Driver")
      .option("dbtable", "breast_cancer_wisconsin")
      .option("user", "root")
      .option("password", "123456")
      .load()

    val frame: DataFrame = df.describe()
    frame.show(false)

    val summary: DataFrame = df.summary()
    summary.show(false)
  }
}
